{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating Results on Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda\\envs\\pytorch\\lib\\site-packages\\numpy\\_distributor_init.py:32: UserWarning: loaded more than 1 DLL from .libs:\n",
      "D:\\anaconda\\envs\\pytorch\\lib\\site-packages\\numpy\\.libs\\libopenblas.QVLO2T66WEPI7JZ63PS3HMOHFEY472BC.gfortran-win_amd64.dll\n",
      "D:\\anaconda\\envs\\pytorch\\lib\\site-packages\\numpy\\.libs\\libopenblas.XWYDX2IKJW2NMTWSFYNGFUWKQU3LYTCZ.gfortran-win_amd64.dll\n",
      "  stacklevel=1)\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "import os\n",
    "import sys\n",
    "import matplotlib.pyplot as plt\n",
    "plt.rcParams['figure.figsize'] = [14, 8]\n",
    "\n",
    "sys.path.append('../..')\n",
    "from pytracking.analysis.plot_results import plot_results, print_results, print_per_sequence_results\n",
    "from pytracking.evaluation import Tracker, get_dataset, trackerlist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots for OTB, NFS and UAV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "Exception",
     "evalue": "Could not read file /home/cx/cx2/OTB100/Basketball/groundtruth_rect.txt",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mException\u001B[0m                                 Traceback (most recent call last)",
      "\u001B[1;32m~\\AppData\\Local\\Temp\\ipykernel_68584\\1434005888.py\u001B[0m in \u001B[0;36m<module>\u001B[1;34m\u001B[0m\n\u001B[0;32m      6\u001B[0m \u001B[0mtrackers\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mextend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mtrackerlist\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m'dimp'\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m'prdimp50'\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mrange\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;36m0\u001B[0m\u001B[1;33m,\u001B[0m\u001B[1;36m5\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;34m'PrDiMP50'\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m      7\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m----> 8\u001B[1;33m \u001B[0mdataset\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mget_dataset\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m'otb'\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m      9\u001B[0m plot_results(trackers, dataset, 'OTB', merge_results=True, plot_types=('success', 'prec'), \n\u001B[0;32m     10\u001B[0m              skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05, exclude_invalid_frames=False)\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\evaluation\\datasets.py\u001B[0m in \u001B[0;36mget_dataset\u001B[1;34m(*args)\u001B[0m\n\u001B[0;32m     58\u001B[0m     \u001B[0mdset\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mSequenceList\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     59\u001B[0m     \u001B[1;32mfor\u001B[0m \u001B[0mname\u001B[0m \u001B[1;32min\u001B[0m \u001B[0margs\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 60\u001B[1;33m         \u001B[0mdset\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mextend\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mload_dataset\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mname\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     61\u001B[0m     \u001B[1;32mreturn\u001B[0m \u001B[0mdset\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\evaluation\\datasets.py\u001B[0m in \u001B[0;36mload_dataset\u001B[1;34m(name)\u001B[0m\n\u001B[0;32m     51\u001B[0m     \u001B[0mm\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mimportlib\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mimport_module\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mdset_info\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mmodule\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     52\u001B[0m     \u001B[0mdataset\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mgetattr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mm\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdset_info\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mclass_name\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m**\u001B[0m\u001B[0mdset_info\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mkwargs\u001B[0m\u001B[1;33m)\u001B[0m  \u001B[1;31m# Call the constructor\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 53\u001B[1;33m     \u001B[1;32mreturn\u001B[0m \u001B[0mdataset\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mget_sequence_list\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     54\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     55\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\evaluation\\otbdataset.py\u001B[0m in \u001B[0;36mget_sequence_list\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m     21\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     22\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0mget_sequence_list\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 23\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mSequenceList\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_construct_sequence\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0ms\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0ms\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msequence_info_list\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     24\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     25\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_construct_sequence\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msequence_info\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\evaluation\\otbdataset.py\u001B[0m in \u001B[0;36m<listcomp>\u001B[1;34m(.0)\u001B[0m\n\u001B[0;32m     21\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     22\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0mget_sequence_list\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 23\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mSequenceList\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;33m[\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0m_construct_sequence\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0ms\u001B[0m\u001B[1;33m)\u001B[0m \u001B[1;32mfor\u001B[0m \u001B[0ms\u001B[0m \u001B[1;32min\u001B[0m \u001B[0mself\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0msequence_info_list\u001B[0m\u001B[1;33m]\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     24\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     25\u001B[0m     \u001B[1;32mdef\u001B[0m \u001B[0m_construct_sequence\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mself\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0msequence_info\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\evaluation\\otbdataset.py\u001B[0m in \u001B[0;36m_construct_sequence\u001B[1;34m(self, sequence_info)\u001B[0m\n\u001B[0;32m     40\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     41\u001B[0m         \u001B[1;31m# NOTE: OTB has some weird annos which panda cannot handle\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 42\u001B[1;33m         \u001B[0mground_truth_rect\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mload_text\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mstr\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0manno_path\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m','\u001B[0m\u001B[1;33m,\u001B[0m \u001B[1;32mNone\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfloat64\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mbackend\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m'numpy'\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     43\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     44\u001B[0m         return Sequence(sequence_info['name'], frames, 'otb', ground_truth_rect[init_omit:,:],\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\utils\\load_text.py\u001B[0m in \u001B[0;36mload_text\u001B[1;34m(path, delimiter, dtype, backend)\u001B[0m\n\u001B[0;32m     37\u001B[0m \u001B[1;32mdef\u001B[0m \u001B[0mload_text\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m' '\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mfloat32\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mbackend\u001B[0m\u001B[1;33m=\u001B[0m\u001B[1;34m'numpy'\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     38\u001B[0m     \u001B[1;32mif\u001B[0m \u001B[0mbackend\u001B[0m \u001B[1;33m==\u001B[0m \u001B[1;34m'numpy'\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 39\u001B[1;33m         \u001B[1;32mreturn\u001B[0m \u001B[0mload_text_numpy\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     40\u001B[0m     \u001B[1;32melif\u001B[0m \u001B[0mbackend\u001B[0m \u001B[1;33m==\u001B[0m \u001B[1;34m'pandas'\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     41\u001B[0m         \u001B[1;32mreturn\u001B[0m \u001B[0mload_text_pandas\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;32mD:\\study\\track\\modal\\modal\\pytracking\\utils\\load_text.py\u001B[0m in \u001B[0;36mload_text_numpy\u001B[1;34m(path, delimiter, dtype)\u001B[0m\n\u001B[0;32m     12\u001B[0m                 \u001B[1;32mpass\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     13\u001B[0m \u001B[1;33m\u001B[0m\u001B[0m\n\u001B[1;32m---> 14\u001B[1;33m         \u001B[1;32mraise\u001B[0m \u001B[0mException\u001B[0m\u001B[1;33m(\u001B[0m\u001B[1;34m'Could not read file {}'\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mformat\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0m\u001B[0;32m     15\u001B[0m     \u001B[1;32melse\u001B[0m\u001B[1;33m:\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n\u001B[0;32m     16\u001B[0m         \u001B[0mground_truth_rect\u001B[0m \u001B[1;33m=\u001B[0m \u001B[0mnp\u001B[0m\u001B[1;33m.\u001B[0m\u001B[0mloadtxt\u001B[0m\u001B[1;33m(\u001B[0m\u001B[0mpath\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdelimiter\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdelimiter\u001B[0m\u001B[1;33m,\u001B[0m \u001B[0mdtype\u001B[0m\u001B[1;33m=\u001B[0m\u001B[0mdtype\u001B[0m\u001B[1;33m)\u001B[0m\u001B[1;33m\u001B[0m\u001B[1;33m\u001B[0m\u001B[0m\n",
      "\u001B[1;31mException\u001B[0m: Could not read file /home/cx/cx2/OTB100/Basketball/groundtruth_rect.txt"
     ]
    }
   ],
   "source": [
    "trackers = []\n",
    "trackers.extend(trackerlist('atom', 'default', range(0,5), 'ATOM'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp18', range(0,5), 'DiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp50', range(0,5), 'DiMP50'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp18', range(0,5), 'PrDiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp50', range(0,5), 'PrDiMP50'))\n",
    "\n",
    "dataset = get_dataset('otb')\n",
    "plot_results(trackers, dataset, 'OTB', merge_results=True, plot_types=('success', 'prec'), \n",
    "             skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05, exclude_invalid_frames=False)\n",
    "\n",
    "dataset = get_dataset('nfs')\n",
    "plot_results(trackers, dataset, 'NFS', merge_results=True, plot_types=('success', 'prec'), \n",
    "             skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05, exclude_invalid_frames=False)\n",
    "\n",
    "dataset = get_dataset('uav')\n",
    "plot_results(trackers, dataset, 'UAV', merge_results=True, plot_types=('success', 'prec'), \n",
    "             skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05, exclude_invalid_frames=False)\n",
    "\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "plot_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, plot_types=('success', 'prec'), \n",
    "             skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05, exclude_invalid_frames=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots for LaSOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trackers = []\n",
    "trackers.extend(trackerlist('atom', 'default', range(0,5), 'ATOM'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp18', range(0,5), 'DiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp50', range(0,5), 'DiMP50'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp18', range(0,5), 'PrDiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp50', range(0,5), 'PrDiMP50'))\n",
    "\n",
    "dataset = get_dataset('lasot')\n",
    "plot_results(trackers, dataset, 'LaSOT', merge_results=True, plot_types=('success'), \n",
    "             skip_missing_seq=False, force_evaluation=True, plot_bin_gap=0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tables for OTB, NFS, UAV and LaSOT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "trackers = []\n",
    "trackers.extend(trackerlist('atom', 'default', range(0,5), 'ATOM'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp18', range(0,5), 'DiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'dimp50', range(0,5), 'DiMP50'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp18', range(0,5), 'PrDiMP18'))\n",
    "trackers.extend(trackerlist('dimp', 'prdimp50', range(0,5), 'PrDiMP50'))\n",
    "\n",
    "dataset = get_dataset('otb')\n",
    "print_results(trackers, dataset, 'OTB', merge_results=True, plot_types=('success', 'prec', 'norm_prec'))\n",
    "\n",
    "dataset = get_dataset('nfs')\n",
    "print_results(trackers, dataset, 'NFS', merge_results=True, plot_types=('success', 'prec', 'norm_prec'))\n",
    "\n",
    "dataset = get_dataset('uav')\n",
    "print_results(trackers, dataset, 'UAV', merge_results=True, plot_types=('success', 'prec', 'norm_prec'))\n",
    "\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "print_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, plot_types=('success', 'prec', 'norm_prec'))\n",
    "\n",
    "dataset = get_dataset('lasot')\n",
    "print_results(trackers, dataset, 'LaSOT', merge_results=True, plot_types=('success', 'prec', 'norm_prec'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filtered per-sequence results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print per sequence results for sequences where all trackers fail, i.e. all trackers have average overlap in percentage of less than 10.0\n",
    "filter_criteria = {'mode': 'ao_max', 'threshold': 10.0}\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "print_per_sequence_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, filter_criteria=filter_criteria, force_evaluation=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print per sequence results for sequences where at least one tracker fails, i.e. a tracker has average overlap in percentage of less than 10.0\n",
    "filter_criteria = {'mode': 'ao_min', 'threshold': 10.0}\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "print_per_sequence_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, filter_criteria=filter_criteria, force_evaluation=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print per sequence results for sequences where the trackers have differing behavior.\n",
    "# i.e. average overlap in percentage for different trackers on a sequence differ by at least 40.0\n",
    "filter_criteria = {'mode': 'delta_ao', 'threshold': 40.0}\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "print_per_sequence_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, filter_criteria=filter_criteria, force_evaluation=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Print per sequence results for all sequences\n",
    "filter_criteria = None\n",
    "dataset = get_dataset('otb', 'nfs', 'uav')\n",
    "print_per_sequence_results(trackers, dataset, 'OTB+NFS+UAV', merge_results=True, filter_criteria=filter_criteria, force_evaluation=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}